import orngText
import html2text
from string import whitespace
import codecs
import re


def getStopWords():
    stopWords = codecs.open("mtestop-sl.wfl", encoding='utf-8')
    return [i.split("\t")[0] for i in stopWords.readlines()]


def preprocess(text):
    p = orngText.Preprocess(language="sl")
    text = html2text.html2text(text).encode("utf-8")
    #text = p.removeStopwords(text)
    return p.lemmatize(text).decode("utf-8").lower()

def removeStopWords(text, stopWords):
    '''Remove words that are shorter then 3'''
    return " ".join(i for i in text.split() if len(i)> 2 and i not in stopWords)

def simpleNormalize(text, title_unicode_trans):
    '''returns only letter and space'''
    text = html2text.html2text(text.lower())
    return re.sub("\s+", " ", text.translate(title_unicode_trans)).lower()

class TitleUnicodeTranslate(dict):
    def __missing__(self,item):
        uni = unichr(item)
        res = u" "
        if uni.isupper() or uni.islower() or uni == " ":
            res = uni
        self[item] = res
        return res

#title_unicode_trans = TitleUnicodeTranslate()
#print simpleNormalize(u"ysdfgs-,. \t\t\td34", title_unicode_trans)